import scrapy
from boss.items import JobItem


class JobSpider(scrapy.Spider):
    "目前先爬取java"
    name = "job"
    path = "https://www.zhipin.com/c101010100-p100101?page=%d"
    page = 1
    start_urls = [
        path % page
    ]

    def parse(self, response):
        # filename = "java" + '.html'
        # with open(filename, 'wb') as f:
        #     f.write(response.body)
        item = JobItem()
        for sel in response.xpath('//div[@id="main"]//li'):
            # 职业
            item['job_name'] = sel.xpath('div/div/h3/a/div[@class="job-title"]/text()').extract()[0]
            # 薪资
            item['pay'] = sel.xpath('div/div/h3/a/span/text()').extract()[0]
            # 城市
            item['city'] = sel.xpath('div/div[@class="info-primary"]/p/text()').extract()[0]
            # 司龄
            item['seniority'] = sel.xpath('div/div[@class="info-primary"]/p/text()').extract()[1]
            # 学历
            item['education'] = sel.xpath('div/div[@class="info-primary"]/p/text()').extract()[2]
            # 公司名称
            item['com_name'] = sel.xpath('div/div[@class="info-company"]//h3/a/text()').extract()[0]
            li = sel.xpath('div/div[@class="info-company"]//p/text()').extract()
            if (len(li) == 2):
                item['stage'] = ""
                item['people_num'] = li[1]
            else:
                # 行业
                item['industry'] = li[0]
                # 公司所处阶段 (a.b轮)
                item['stage'] = li[1]
                # 公司人数
                item['people_num'] = li[2]
            # item['jobDescribe'] = sel.xpath('div/div/h3/a/div/text()').extract()
            yield item
            print(item)

        if self.page < 10:
            self.page += 1
            yield scrapy.Request(self.path % self.page, callback=self.parse)

